/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.alag.ci.textanalysis.impl;

import com.alag.ci.MetaDataVector;
import java.util.*;

/**
 *
 * @author Justina
 */
public class SimpleBiTermStopWordStemmerMetaDataExtractor extends SimpleStopWordStemmerMetaDataExtractor {
    
    @Override
    protected MetaDataVector getMetaDataVector(String text) {
        Map<String, Integer> keywordMap = new HashMap<String, Integer>();
        List<String> allTokens = new ArrayList<String>();
        StringTokenizer st = new StringTokenizer(text);
        while (st.hasMoreTokens()) {
            String token = normalizeToken(st.nextToken());
            if (acceptToken(token)) {
                Integer count = keywordMap.get(token);
                if (count == null) {
                    count = new Integer(0);
                }
                count ++;
                keywordMap.put(token, count);
                allTokens.add(token);
            }
        }
        
        String firstToken = allTokens.get(0);
        for (String token: allTokens) {
            String biTerm = firstToken + " " + token;
            if (isValidBiTermToken(biTerm)) {
                Integer count = keywordMap.get(biTerm);
                if (count == null) {
                    count = new Integer(0);
                }
                count ++;
                keywordMap.put(biTerm, count);
                keywordMap.remove(firstToken);
                keywordMap.remove(token);
            }
            firstToken = token;
        }
        MetaDataVector mdv = createMetaDataVector(keywordMap);
        return mdv;
    }

    private boolean isValidBiTermToken(String biTerm) {
        if ("collective intelligence".equals(biTerm)) {
            return true;
        }
        return false;
    }
    
}
